package de.distMLP.train;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;

import de.distMLP.data.TrainingExample;
import de.distMLP.messages.MatrixArrayWritable;
import de.mlp_distributed.mlp.core.MultiLayerPerceptron;
import de.mlp_distributed.mlp.math.Factory;
import de.mlp_distributed.mlp.math.mahout.Matrix;
import de.mlp_distributed.mlp.math.mahout.MatrixWritable;
import de.mlp_distributed.mlp.math.mahout.RandomAccessSparseVector;
import de.mlp_distributed.mlp.math.mahout.Vector;

public class Training_Helper {

	private Training_Helper() {
		super();
	}

	protected static void divideMatrixByDouble(final Matrix[] weightChanges, final double value) {
		for (int i = 0; i < weightChanges.length; i++) {
			weightChanges[i] = weightChanges[i].divide(value);
		}
	}

	protected static Matrix[] calculateGradient(final TrainingExample ex, final MultiLayerPerceptron mlp) {
		return mlp.getDerivativeOfTheCostWithoutRegularization(ex.getFeatureVector(), ex.getTargetVector());
	}

	protected static Matrix[] getCopyOfMatrixArray(final Matrix[] m) {
		final Matrix[] copy = new Matrix[m.length];
		for (int i = 0; i < m.length; i++) {
			copy[i] = Factory.getInstance().getMatrixFactory().construct(m[i].rowSize(), m[i].columnSize());
			for (int column = 0; column < copy[i].columnSize(); column++) {
				for (int row = 0; row < copy[i].rowSize(); row++) {
					copy[i].setQuick(row, column, new Double(m[i].get(row, column)));
				}
			}
		}
		return copy;
	}

	/**
	 * Input: 0:0.0 |1:0.0 0:0.0 left part contains target values e.g. class.
	 * right part contains feature values.
	 * 
	 * @param value
	 * @param featureVectorSize
	 * @return
	 */
	public static Vector getTargetVectorFromTrainingsdata(final Text value, final int featureVectorSize) {
		final int locationOfPipe = value.toString().indexOf("|");
		final Vector targetVector = new RandomAccessSparseVector(featureVectorSize);
		final String targetValues = value.toString().substring(0, locationOfPipe);
		final StringTokenizer targetTokens = new StringTokenizer(targetValues.toString(), " ");
		while (targetTokens.hasMoreTokens()) {
			final String targetToken = targetTokens.nextToken();
			final String[] targetSplit = targetToken.split(":");
			final int outputnb = Integer.parseInt(targetSplit[0]);
			final double target = Double.parseDouble(targetSplit[1]);
			targetVector.setQuick(outputnb, target);
		}
		return targetVector;
	}

	/**
	 * Input: 0:0.0 |1:0.0 0:0.0 left part contains target values. right part
	 * contains feature values.
	 * 
	 * @param value
	 * @param featureVectorSize
	 * @return
	 */
	public static Vector getFeatureVectorFromTrainingsdata(final Text value, final int featureVectorSize) {
		final int locationOfPipe = value.toString().indexOf("|");
		final Vector featureVector = new RandomAccessSparseVector(featureVectorSize);
		final String featureValues = value.toString().substring(locationOfPipe + 1, value.toString().length());
		final StringTokenizer featureTokens = new StringTokenizer(featureValues.toString(), " ");
		while (featureTokens.hasMoreTokens()) {
			final String featureToken = featureTokens.nextToken();
			final String[] featureSplit = featureToken.split(":");
			final int feature = Integer.parseInt(featureSplit[0]);
			final double feature_value = Double.parseDouble(featureSplit[1]);
			featureVector.setQuick(feature, feature_value);
		}
		return featureVector;
	}

	public static MultiLayerPerceptron readNetwork(final Configuration configuration, final Path filenamePath) throws IOException {
		final MultiLayerPerceptron mlp;
		final boolean localfs = Training_Helper.isLocalFileSystem(configuration);
		if (!localfs) {
			final FileSystem fs = FileSystem.get(configuration);
			final FSDataInputStream in = fs.open(filenamePath);
			mlp = MultiLayerPerceptron.createMultiLayerPerceptron(in);
		} else {
			final FileInputStream fos = new FileInputStream(filenamePath.toString());
			final DataInputStream in = new DataInputStream(fos);
			mlp = MultiLayerPerceptron.createMultiLayerPerceptron(in);
		}
		return mlp;
	}

	protected static void writeNetwork(final Configuration configuration, final Path filenamePath, final MultiLayerPerceptron mlp)
			throws IOException, URISyntaxException {
		final boolean localfs = Training_Helper.isLocalFileSystem(configuration);

		final FileSystem fs = FileSystem.get(configuration);
		if (fs.exists(filenamePath)) {
			// remove the file first
			fs.delete(filenamePath, true);
		}

		if (!localfs) {
			final FileSystem hdfs = FileSystem.get(new java.net.URI("hdfs://localhost:54310"), configuration);
			if (fs.exists(filenamePath)) {
				// remove the file first
				fs.delete(filenamePath, true);
			}
			final Path file = new Path("hdfs://localhost:54310" + filenamePath.getName());
			final FSDataOutputStream out = fs.create(file);
			mlp.write(out);
		} else {
			final File f = new File(filenamePath.toString());
			f.getParentFile().mkdirs();
			final FileOutputStream fos = new FileOutputStream(filenamePath.toString());
			final DataOutputStream dos = new DataOutputStream(fos);
			mlp.write(dos);
		}
	}

	public static boolean isLocalFileSystem(final Configuration configuration) {
		final boolean localfs = (configuration.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
		return localfs;
	}

	protected static MatrixArrayWritable createMatrixArrayWritable(final Matrix[] matrix) {
		final MatrixWritable[] list = new MatrixWritable[matrix.length];
		for (int i = 0; i < matrix.length; i++) {
			list[i] = new MatrixWritable(matrix[i]);
		}
		return new MatrixArrayWritable(list);
	}

	protected static Matrix[] getMatricesFromMatrixArrayWritable(final MatrixArrayWritable mar) {
		final Matrix[] list = new Matrix[mar.get().length];
		for (int i = 0; i < mar.get().length; i++) {
			final MatrixWritable tmp = (MatrixWritable) mar.get()[i];
			list[i] = tmp.get();
		}
		return list;
	}

	protected static int[] getNumberOfHiddenUnits(final Configuration conf) {
		final int nbHiddenLayer = conf.getInt(de.distMLP.train.Configuration.NB_HIDDEN_LAYERS, 0);
		final int[] hiddenLayerNeuronCount = new int[nbHiddenLayer];
		for (int i = 0; i < nbHiddenLayer; i++) {
			hiddenLayerNeuronCount[i] = conf.getInt(de.distMLP.train.Configuration.NB_UNITS_HIDDEN_LAYER + (i + 1), 0);
		}
		return hiddenLayerNeuronCount;
	}

	/**
	 * Adds delta w to weight matrix.
	 * 
	 * @param weightMatrix
	 *            matrix, data is added to.
	 * @param deltaWs
	 *            matrix, containing the data to add.
	 */
	protected static void addWeightChanges(final Matrix[] weightMatrix, final Matrix[] deltaWs) {
		// Sum up the weightChanges
		for (int i = 0; i < weightMatrix.length; i++) {
			if (weightMatrix[i] == null) {
				weightMatrix[i] = deltaWs[i];
			} else {
				weightMatrix[i] = weightMatrix[i].plus(deltaWs[i]);
			}
		}
	}
}
